# -*- coding: utf-8 -*-
#!/usr/lib/python3.5

import scrapy
import re
import abc
import hashlib
from snowball_crawler.items import SnowballCubeItem
#from snowball_crawler.utils import md5

class snowball_base_spider(scrapy.Spider):
    '''
    雪球爬虫基类 实现自动登录
    '''
    name ="snowball"
    allowed_domains=["xueqiu.com"]

    agent = 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'

    headers = {
        'User-Agent': agent,
        'Host': "xueqiu.com",
        "Accept": "*/*",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "zh-CN,zh;q=0.8",
        "Connection": "keep-alive"
    }

    def get_md5(text):
        md5 = hashlib.md5()
        md5.update(text.encode())
        return md5.hexdigest().upper()

    userdata = {
        "password": get_md5('abc****56'),
        "remember_me": "on",
        "telephone": '183****6896'
    }

    def start_requests(self): # 访问首页产生 cookies
        self.logger.debug('start request')
        return [scrapy.Request(url='https://xueqiu.com/',
                               headers = self.headers,
                               callback = self.before_login)]

    def before_login(self, response): # 模拟更真实的登录
        return [scrapy.Request(url="https://xueqiu.com/service/csrf?api=%2Fuser%2Flogin",
                               headers= self.headers,
                               callback = self.do_login)]

    def do_login(self, response): #提交登录
        login_url = "https://xueqiu.com/user/login"

        return [scrapy.FormRequest(url=login_url,
                               headers=self.headers,
                               formdata= self.userdata,
                               dont_filter=True,
                               callback=self.parse_login)]

    def parse_login(self, response):
        self.logger.debug('parselogin url=%s ' % response.url)
        pa = r'https://xueqiu.com/(\d+)'       # 登录成功会url会是 https://xueqiu.com/17232323 数字代表userID 用正则表达式解析
        res = re.findall(pa, response.url)
        if res == []:
            self.logger.warn('登录失败')
        else:
            self.logger.info('登录成功，ID=%s' % res[0])
            return self.start_spider()

    @abc.abstractmethod
    def start_spider(self):
        self.logger.debug('base login done ')
        return
